Transform Weather Sensor Data from Waggle to an Xarray Dataset + Plot with ACT

Imports

import sage_data_client
from bokeh.models.formatters import DatetimeTickFormatter
import hvplot.pandas
import hvplot.xarray
import holoviews as hv
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import xarray as xr
import matplotlib.pyplot as plt
from metpy.plots import USCOUNTIES
import act
import numpy as np
import pandas as pd
import warnings
from bokeh.models import DatetimeTickFormatter

def apply_formatter(plot, element):
    plot.handles['xaxis'].formatter = DatetimeTickFormatter(hours='%m/%d/%Y \n %H:%M',
                                                            minutes='%m/%d/%Y \n %H:%M',
                                                            hourmin='%m/%d/%Y \n %H:%M',
                                                            days='%m/%d/%Y \n %H:%M',
                                                            months='%m/%d/%Y \n %H:%M')
xr.set_options(keep_attrs=True)
warnings.filterwarnings("ignore")
hv.extension("bokeh")

Query for the available Data

wxt_df = sage_data_client.query(
    start="-3h",
    filter={
        "sensor": "vaisala-wxt536"
    }
)

Exception ignored in: <bound method IPythonKernel._clean_thread_parent_frames of <ipykernel.ipkernel.IPythonKernel object at 0x7f700a9fceb0>>
Traceback (most recent call last):
  File "/home/runner/miniconda3/envs/instrument-cookbooks-dev/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 775, in _clean_thread_parent_frames
    def _clean_thread_parent_frames(
KeyboardInterrupt: 

Configure Helper Functions and Renaming Conventions

The renaming is required due to . notations being problematic when working with both Pandas and Xarray data structures.

variable_rename_dict = {'wxt.env.humidity':'air_humidity',
                        'wxt.env.pressure':'air_pressure',
                        'wxt.env.temp':'air_temperature',
                        'wxt.heater.temp':'heater_temperature',
                        'wxt.heater.volt':'heater_voltage',
                        'wxt.rain.accumulation':'rain_accumulation',
                        'wxt.wind.direction':'wind_direction',
                        'wxt.wind.speed':'wind_speed',
                        'sys.gps.lat':'latitude',
                        'sys.gps.lon':'longitude',
                    }

def generate_data_array(df, variable, rename_variable_dict=variable_rename_dict):
    new_variable_name = rename_variable_dict[variable]
    df_variable= df.loc[df.name == variable]
    ds = df_variable.to_xarray().rename({'value':new_variable_name,
                                         'timestamp':'time',
                                         'meta.vsn':'node'})
    ds[new_variable_name].attrs['units'] = df_variable['meta.units'].values[0]
    ds['time'] = pd.to_datetime(ds.time)
    ds.attrs['datastream'] = ds.node.values[0]
    return ds[[new_variable_name]]

def generate_dataset(df, variables, rename_variable_dict=variable_rename_dict):
    reindexed = df.set_index(['meta.vsn', 'timestamp'])
    return xr.merge([generate_data_array(reindexed, variable) for variable in variables])

Transform the Data to Xarray

wxt_variables = wxt_df.name.unique()
wxt_variables

array(['wxt.env.humidity', 'wxt.env.pressure', 'wxt.env.temp',
       'wxt.heater.temp', 'wxt.heater.volt', 'wxt.rain.accumulation',
       'wxt.wind.direction', 'wxt.wind.speed'], dtype=object)

wxt_ds = generate_dataset(wxt_df, wxt_variables).squeeze()
wxt_ds

<xarray.Dataset>
Dimensions:             (time: 134566)
Coordinates:
    node                <U4 'W057'
  * time                (time) datetime64[ns] 2023-04-14T17:59:43.475266934 ....
Data variables:
    air_humidity        (time) float64 35.2 35.2 35.2 35.2 ... nan nan nan nan
    air_pressure        (time) float64 981.4 981.4 981.4 ... 980.1 980.1 980.1
    air_temperature     (time) float64 24.9 24.9 24.9 24.9 ... 26.3 26.3 26.3
    heater_temperature  (time) float64 30.9 30.9 30.9 30.9 ... 30.8 30.8 30.8
    heater_voltage      (time) float64 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
    rain_accumulation   (time) float64 0.03 0.03 0.03 0.03 ... 0.03 0.03 0.03
    wind_direction      (time) float64 181.0 181.0 185.0 ... 163.0 163.0 163.0
    wind_speed          (time) float64 2.9 2.9 2.8 2.8 2.8 ... 2.7 2.8 2.8 2.8
Attributes:
    datastream:  W057

xarray.Dataset

• Dimensions:
  • time: 134566
• Coordinates: (2)
  • node
    ()
    <U4
    'W057'

    array('W057', dtype='<U4')

  • time
    (time)
    datetime64[ns]
    2023-04-14T17:59:43.475266934 .....

    array(['2023-04-14T17:59:43.475266934', '2023-04-14T17:59:43.549903215',
           '2023-04-14T17:59:43.718253933', ..., '2023-04-14T20:59:43.596753288',
           '2023-04-14T20:59:43.672562546', '2023-04-14T20:59:43.748872004'],
          dtype='datetime64[ns]')

• Data variables: (8)
  • air_humidity
    (time)
    float64
    35.2 35.2 35.2 35.2 ... nan nan nan

    units :

    percent

    array([35.2, 35.2, 35.2, ...,  nan,  nan,  nan])

  • air_pressure
    (time)
    float64
    981.4 981.4 981.4 ... 980.1 980.1

    units :

    hectoPascal

    array([981.4, 981.4, 981.4, ..., 980.1, 980.1, 980.1])

  • air_temperature
    (time)
    float64
    24.9 24.9 24.9 ... 26.3 26.3 26.3

    units :

    degree Celsius

    array([24.9, 24.9, 24.9, ..., 26.3, 26.3, 26.3])

  • heater_temperature
    (time)
    float64
    30.9 30.9 30.9 ... 30.8 30.8 30.8

    units :

    degree Celsius

    array([30.9, 30.9, 30.9, ..., 30.8, 30.8, 30.8])

  • heater_voltage
    (time)
    float64
    0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0

    units :

    volts

    array([0., 0., 0., ..., 0., 0., 0.])

  • rain_accumulation
    (time)
    float64
    0.03 0.03 0.03 ... 0.03 0.03 0.03

    units :

    milimeters

    array([0.03, 0.03, 0.03, ..., 0.03, 0.03, 0.03])

  • wind_direction
    (time)
    float64
    181.0 181.0 185.0 ... 163.0 163.0

    units :

    degrees

    array([181., 181., 185., ..., 163., 163., 163.])

  • wind_speed
    (time)
    float64
    2.9 2.9 2.8 2.8 ... 2.7 2.8 2.8 2.8

    units :

    meters per second

    array([2.9, 2.9, 2.8, ..., 2.8, 2.8, 2.8])

• Indexes: (1)
  • time
    PandasIndex

    PandasIndex(DatetimeIndex(['2023-04-14 17:59:43.475266934',
                   '2023-04-14 17:59:43.549903215',
                   '2023-04-14 17:59:43.718253933',
                   '2023-04-14 17:59:43.793684050',
                   '2023-04-14 17:59:43.869524062',
                   '2023-04-14 17:59:43.951531819',
                   '2023-04-14 17:59:44.026595627',
                   '2023-04-14 17:59:44.102201587',
                   '2023-04-14 17:59:44.177617720',
                   '2023-04-14 17:59:44.252600150',
                   ...
                   '2023-04-14 20:59:43.065794079',
                   '2023-04-14 20:59:43.140582550',
                   '2023-04-14 20:59:43.217408754',
                   '2023-04-14 20:59:43.292724659',
                   '2023-04-14 20:59:43.367861968',
                   '2023-04-14 20:59:43.442852907',
                   '2023-04-14 20:59:43.518273645',
                   '2023-04-14 20:59:43.596753288',
                   '2023-04-14 20:59:43.672562546',
                   '2023-04-14 20:59:43.748872004'],
                  dtype='datetime64[ns]', name='time', length=134566, freq=None))

• Attributes: (1)

  datastream :

  W057

Resample the data to minute requency

minute_ds = wxt_ds.resample(time='1T').mean()

Visualize using hvplot

meteogram_variables = ['air_temperature', 'air_humidity', 'wind_speed', 'wind_direction']
plots = []
for variable in meteogram_variables:
    plots.append(wxt_ds[variable].hvplot.line(label='10 Hz Data') * 
                 minute_ds[variable].hvplot.line(label='1 Minute Data'))
hv.Layout(plots).cols(2)

Plot Using the Atmospheric Data Community Toolkit (ACT)

WindDisplay = act.plotting.WindRoseDisplay(minute_ds, figsize=(8, 10), subplot_shape=(1,))
WindDisplay.plot(
    'wind_direction', 'wind_speed', spd_bins=np.linspace(0, 25, 5), num_dirs=30, tick_interval=2, subplot_index=(0,)
)

<PolarAxes: title={'center': 'W057 on 20230414'}>